# coding:utf8

from bs4 import BeautifulSoup
import re
from urllib import parse

# import urlparse

class HtmlParser(object):
	"""docstring for HtmlParser"""
	def __init__(self):
		print("HtmlParser...")

	def get_new_urls(self, page_url, soup):
		new_urls = set()
		links = soup.find_all('a', href=re.compile(r"/item/.*"))

		for link in links:
			new_url = link['href']
			full_url = parse.urljoin(page_url, new_url)
			new_urls.add(full_url)
		return new_urls

	def get_new_data(self, page_url, soup):
		new_data = {}
		new_data['url'] = page_url

		title_node = soup.find('dd', class_='lemmaWgt-lemmaTitle-title').find('h1')
		new_data['title'] = title_node.get_text()

		summary_node = soup.find('div', class_='lemma-summary')
		new_data['summary'] = summary_node.get_text()

		summary_pic_node = soup.find('div', class_='summary-pic').find('img')
		new_data['summary_pic'] = summary_pic_node['src']

		return new_data

	def parse(self, page_url, html_content):
		if page_url is None or html_content is None:
			return
		soup = BeautifulSoup(html_content, 'html.parser')

		# print(soup)

		new_urls = self.get_new_urls(page_url, soup)
		new_data = self.get_new_data(page_url, soup)

		return new_urls, new_data